{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Copyright 2020 NVIDIA Corporation. All Rights Reserved.\n",
    "#\n",
    "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
    "# you may not use this file except in compliance with the License.\n",
    "# You may obtain a copy of the License at\n",
    "#\n",
    "#     http://www.apache.org/licenses/LICENSE-2.0\n",
    "#\n",
    "# Unless required by applicable law or agreed to in writing, software\n",
    "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
    "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
    "# See the License for the specific language governing permissions and\n",
    "# limitations under the License.\n",
    "# =============================================================================="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/numba/cuda/envvars.py:17: NumbaWarning: \n",
      "Environment variables with the 'NUMBAPRO' prefix are deprecated and consequently ignored, found use of NUMBAPRO_NVVM=/usr/local/cuda/nvvm/lib64/libnvvm.so.\n",
      "\n",
      "For more information about alternatives visit: ('https://numba.pydata.org/numba-doc/latest/cuda/overview.html', '#cudatoolkit-lookup')\n",
      "  warnings.warn(errors.NumbaWarning(msg))\n",
      "/opt/conda/lib/python3.7/site-packages/numba/cuda/envvars.py:17: NumbaWarning: \n",
      "Environment variables with the 'NUMBAPRO' prefix are deprecated and consequently ignored, found use of NUMBAPRO_LIBDEVICE=/usr/local/cuda/nvvm/libdevice/.\n",
      "\n",
      "For more information about alternatives visit: ('https://numba.pydata.org/numba-doc/latest/cuda/overview.html', '#cudatoolkit-lookup')\n",
      "  warnings.warn(errors.NumbaWarning(msg))\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "from time import time\n",
    "import re\n",
    "import glob\n",
    "import warnings\n",
    "\n",
    "# tools for data preproc/loading\n",
    "import torch\n",
    "import rmm\n",
    "import nvtabular as nvt\n",
    "from nvtabular.ops import Normalize,  Categorify,  LogOp, FillMissing, Clip, get_embedding_sizes\n",
    "from nvtabular.loader.torch import TorchAsyncItr, DLDataLoader\n",
    "from nvtabular.utils import device_mem_size\n",
    "\n",
    "# tools for training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define some information about where to get our data\n",
    "INPUT_DATA_DIR = os.environ.get('INPUT_DATA_DIR', '/raid/criteo/tests/crit_int_pq')\n",
    "OUTPUT_DATA_DIR = os.environ.get('OUTPUT_DATA_DIR', '/raid/criteo/tests/test_dask') # where we'll save our procesed data to\n",
    "BATCH_SIZE = int(os.environ.get('BATCH_SIZE', 32768))\n",
    "AMP = os.environ.get(\"AMP\", \"true\") \n",
    "AMP = True if AMP.lower() in \"true\" else False\n",
    "PARTS_PER_CHUNK = int(os.environ.get('PARTS_PER_CHUNK', 2))\n",
    "SHUFFLE = os.environ.get(\"SHUFFLE\", False)\n",
    "NUM_TRAIN_DAYS = 23 # number of days worth of data to use for training, the rest will be used for validation\n",
    "\n",
    "# define our dataset schema\n",
    "CONTINUOUS_COLUMNS = ['I' + str(x) for x in range(1,14)]\n",
    "CATEGORICAL_COLUMNS =  ['C' + str(x) for x in range(1,27)]\n",
    "LABEL_COLUMNS = ['label']\n",
    "COLUMNS = CONTINUOUS_COLUMNS + CATEGORICAL_COLUMNS + LABEL_COLUMNS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "output_train_dir = os.path.join(OUTPUT_DATA_DIR, 'train/')\n",
    "output_valid_dir = os.path.join(OUTPUT_DATA_DIR, 'valid/')\n",
    "! mkdir -p $output_train_dir\n",
    "! mkdir -p $output_valid_dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# rmm.reinitialize(pool_allocator=True, initial_pool_size=0.2 * device_mem_size(kind='free'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_paths = glob.glob(os.path.join(output_train_dir, \"*.parquet\"))\n",
    "valid_paths = glob.glob(os.path.join(output_valid_dir, \"*.parquet\"))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nvtabular.loader.torch import TorchAsyncItr\n",
    "train_data = nvt.Dataset(train_paths[:1], engine=\"parquet\", part_mem_fraction=0.04/PARTS_PER_CHUNK)\n",
    "valid_data = nvt.Dataset(valid_paths[:1], engine=\"parquet\", part_mem_fraction=0.04/PARTS_PER_CHUNK)\n",
    "train_loader = TorchAsyncItr(\n",
    "    train_data,\n",
    "    batch_size=BATCH_SIZE,\n",
    "    cats=CATEGORICAL_COLUMNS,\n",
    "    conts=CONTINUOUS_COLUMNS,\n",
    "    labels=LABEL_COLUMNS,\n",
    "    parts_per_chunk=PARTS_PER_CHUNK,\n",
    "    shuffle=SHUFFLE,\n",
    ")\n",
    "valid_loader = TorchAsyncItr(\n",
    "    valid_data,\n",
    "    batch_size=BATCH_SIZE,\n",
    "    cats=CATEGORICAL_COLUMNS,\n",
    "    conts=CONTINUOUS_COLUMNS,\n",
    "    labels=LABEL_COLUMNS,\n",
    "    parts_per_chunk=PARTS_PER_CHUNK,\n",
    "    shuffle=SHUFFLE,\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "EMBEDDING_DROPOUT_RATE = 0.04\n",
    "DROPOUT_RATES = [0.001, 0.01]\n",
    "HIDDEN_DIMS = [1000, 500]\n",
    "LEARNING_RATE = 0.001\n",
    "EPOCHS = 1\n",
    "embeddings = {'C1': (7599500, 16),\n",
    " 'C10': (5345303, 16),\n",
    " 'C11': (561810, 16),\n",
    " 'C12': (242827, 16),\n",
    " 'C13': (11, 6),\n",
    " 'C14': (2209, 16),\n",
    " 'C15': (10616, 16),\n",
    " 'C16': (100, 16),\n",
    " 'C17': (4, 3),\n",
    " 'C18': (968, 16),\n",
    " 'C19': (15, 7),\n",
    " 'C2': (33521, 16),\n",
    " 'C20': (7838519, 16),\n",
    " 'C21': (2580502, 16),\n",
    " 'C22': (6878028, 16),\n",
    " 'C23': (298771, 16),\n",
    " 'C24': (11951, 16),\n",
    " 'C25': (97, 16),\n",
    " 'C26': (35, 12),\n",
    " 'C3': (17022, 16),\n",
    " 'C4': (7339, 16),\n",
    " 'C5': (20046, 16),\n",
    " 'C6': (4, 3),\n",
    " 'C7': (7068, 16),\n",
    " 'C8': (1377, 16),\n",
    " 'C9': (63, 16)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nvtabular.framework_utils.torch.models import Model\n",
    "from nvtabular.framework_utils.torch.utils import process_epoch\n",
    "model = Model(\n",
    "    embedding_table_shapes=embeddings,\n",
    "    num_continuous=len(CONTINUOUS_COLUMNS),\n",
    "    emb_dropout=EMBEDDING_DROPOUT_RATE,\n",
    "    layer_hidden_dims=HIDDEN_DIMS,\n",
    "    layer_dropout_rates=DROPOUT_RATES,\n",
    ").to('cuda')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = torch.optim.Adam(model.parameters(), lr=LEARNING_RATE)\n",
    "def rmspe_func(y_pred, y):\n",
    "    \"Return y_pred and y to non-log space and compute RMSPE\"\n",
    "    y_pred, y = torch.exp(y_pred) - 1, torch.exp(y) - 1\n",
    "    pct_var = (y_pred - y) / y\n",
    "    return (pct_var**2).mean().pow(0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def batch_transform(batch, cont_cols=CONTINUOUS_COLUMNS, cat_cols=CATEGORICAL_COLUMNS, label_cols=LABEL_COLUMNS):\n",
    "    x_cat = torch.tensor(batch[sorted(cat_cols)].values).type(torch.LongTensor).cuda() \n",
    "    x_cont = torch.tensor(batch[cont_cols].values).type(torch.FloatTensor).cuda()\n",
    "    y = torch.tensor(batch[label_cols[0]].values).type(torch.FloatTensor).cuda()\n",
    "    return x_cat, x_cont, y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> \u001b[0;32m/jp_docs/nvtabular/nvtabular/loader/torch.py\u001b[0m(97)\u001b[0;36m_to_tensor\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32m     95 \u001b[0;31m    \u001b[0;32mdef\u001b[0m \u001b[0m_to_tensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgdf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[0;32m     96 \u001b[0;31m        \u001b[0;32mimport\u001b[0m \u001b[0mpdb\u001b[0m\u001b[0;34m;\u001b[0m \u001b[0mpdb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_trace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[0;32m---> 97 \u001b[0;31m        \u001b[0;32mif\u001b[0m \u001b[0mgdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mempty\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[0;32m     98 \u001b[0;31m            \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[0;32m     99 \u001b[0;31m        \u001b[0mdl_pack\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_dlpack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "ipdb>  n\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> \u001b[0;32m/jp_docs/nvtabular/nvtabular/loader/torch.py\u001b[0m(99)\u001b[0;36m_to_tensor\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32m     97 \u001b[0;31m        \u001b[0;32mif\u001b[0m \u001b[0mgdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mempty\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[0;32m     98 \u001b[0;31m            \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[0;32m---> 99 \u001b[0;31m        \u001b[0mdl_pack\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_dlpack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[0;32m    100 \u001b[0;31m        \u001b[0mtens\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfrom_dlpack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdl_pack\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[0;32m    101 \u001b[0;31m        \u001b[0;32mreturn\u001b[0m \u001b[0mtens\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "ipdb>  n\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> \u001b[0;32m/jp_docs/nvtabular/nvtabular/loader/torch.py\u001b[0m(100)\u001b[0;36m_to_tensor\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32m     98 \u001b[0;31m            \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[0;32m     99 \u001b[0;31m        \u001b[0mdl_pack\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_dlpack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[0;32m--> 100 \u001b[0;31m        \u001b[0mtens\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfrom_dlpack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdl_pack\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[0;32m    101 \u001b[0;31m        \u001b[0;32mreturn\u001b[0m \u001b[0mtens\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[0;32m    102 \u001b[0;31m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "ipdb>  n\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> \u001b[0;32m/jp_docs/nvtabular/nvtabular/loader/torch.py\u001b[0m(101)\u001b[0;36m_to_tensor\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32m     99 \u001b[0;31m        \u001b[0mdl_pack\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_dlpack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[0;32m    100 \u001b[0;31m        \u001b[0mtens\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfrom_dlpack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdl_pack\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[0;32m--> 101 \u001b[0;31m        \u001b[0;32mreturn\u001b[0m \u001b[0mtens\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[0;32m    102 \u001b[0;31m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[0;32m    103 \u001b[0;31m    \u001b[0;31m# TODO: do we need casting or can we replace this with\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "ipdb>  n\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--Return--\n",
      "tensor([[1353...vice='cuda:0')\n",
      "> \u001b[0;32m/jp_docs/nvtabular/nvtabular/loader/torch.py\u001b[0m(101)\u001b[0;36m_to_tensor\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32m     99 \u001b[0;31m        \u001b[0mdl_pack\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_dlpack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[0;32m    100 \u001b[0;31m        \u001b[0mtens\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfrom_dlpack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdl_pack\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[0;32m--> 101 \u001b[0;31m        \u001b[0;32mreturn\u001b[0m \u001b[0mtens\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[0;32m    102 \u001b[0;31m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[0;32m    103 \u001b[0;31m    \u001b[0;31m# TODO: do we need casting or can we replace this with\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "ipdb>  c\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> \u001b[0;32m/jp_docs/nvtabular/nvtabular/loader/torch.py\u001b[0m(97)\u001b[0;36m_to_tensor\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32m     95 \u001b[0;31m    \u001b[0;32mdef\u001b[0m \u001b[0m_to_tensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgdf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[0;32m     96 \u001b[0;31m        \u001b[0;32mimport\u001b[0m \u001b[0mpdb\u001b[0m\u001b[0;34m;\u001b[0m \u001b[0mpdb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_trace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[0;32m---> 97 \u001b[0;31m        \u001b[0;32mif\u001b[0m \u001b[0mgdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mempty\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[0;32m     98 \u001b[0;31m            \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[0;32m     99 \u001b[0;31m        \u001b[0mdl_pack\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_dlpack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "ipdb>  n\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> \u001b[0;32m/jp_docs/nvtabular/nvtabular/loader/torch.py\u001b[0m(99)\u001b[0;36m_to_tensor\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32m     97 \u001b[0;31m        \u001b[0;32mif\u001b[0m \u001b[0mgdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mempty\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[0;32m     98 \u001b[0;31m            \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[0;32m---> 99 \u001b[0;31m        \u001b[0mdl_pack\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_dlpack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[0;32m    100 \u001b[0;31m        \u001b[0mtens\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfrom_dlpack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdl_pack\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[0;32m    101 \u001b[0;31m        \u001b[0;32mreturn\u001b[0m \u001b[0mtens\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "ipdb>  n\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> \u001b[0;32m/jp_docs/nvtabular/nvtabular/loader/torch.py\u001b[0m(100)\u001b[0;36m_to_tensor\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32m     98 \u001b[0;31m            \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[0;32m     99 \u001b[0;31m        \u001b[0mdl_pack\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_dlpack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[0;32m--> 100 \u001b[0;31m        \u001b[0mtens\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfrom_dlpack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdl_pack\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[0;32m    101 \u001b[0;31m        \u001b[0;32mreturn\u001b[0m \u001b[0mtens\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[0;32m    102 \u001b[0;31m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "ipdb>  n\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> \u001b[0;32m/jp_docs/nvtabular/nvtabular/loader/torch.py\u001b[0m(101)\u001b[0;36m_to_tensor\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32m     99 \u001b[0;31m        \u001b[0mdl_pack\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_dlpack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[0;32m    100 \u001b[0;31m        \u001b[0mtens\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfrom_dlpack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdl_pack\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[0;32m--> 101 \u001b[0;31m        \u001b[0;32mreturn\u001b[0m \u001b[0mtens\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[0;32m    102 \u001b[0;31m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[0;32m    103 \u001b[0;31m    \u001b[0;31m# TODO: do we need casting or can we replace this with\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "ipdb>  n\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--Return--\n",
      "<Tensor instance at 0x7f4640a4d230>\n",
      "> \u001b[0;32m/jp_docs/nvtabular/nvtabular/loader/torch.py\u001b[0m(101)\u001b[0;36m_to_tensor\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32m     99 \u001b[0;31m        \u001b[0mdl_pack\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_dlpack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[0;32m    100 \u001b[0;31m        \u001b[0mtens\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfrom_dlpack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdl_pack\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[0;32m--> 101 \u001b[0;31m        \u001b[0;32mreturn\u001b[0m \u001b[0mtens\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[0;32m    102 \u001b[0;31m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[0;32m    103 \u001b[0;31m    \u001b[0;31m# TODO: do we need casting or can we replace this with\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "ipdb>  c\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> \u001b[0;32m/jp_docs/nvtabular/nvtabular/loader/torch.py\u001b[0m(97)\u001b[0;36m_to_tensor\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32m     95 \u001b[0;31m    \u001b[0;32mdef\u001b[0m \u001b[0m_to_tensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgdf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[0;32m     96 \u001b[0;31m        \u001b[0;32mimport\u001b[0m \u001b[0mpdb\u001b[0m\u001b[0;34m;\u001b[0m \u001b[0mpdb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_trace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[0;32m---> 97 \u001b[0;31m        \u001b[0;32mif\u001b[0m \u001b[0mgdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mempty\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[0;32m     98 \u001b[0;31m            \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[0;32m     99 \u001b[0;31m        \u001b[0mdl_pack\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_dlpack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "ipdb>  n\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> \u001b[0;32m/jp_docs/nvtabular/nvtabular/loader/torch.py\u001b[0m(99)\u001b[0;36m_to_tensor\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32m     97 \u001b[0;31m        \u001b[0;32mif\u001b[0m \u001b[0mgdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mempty\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[0;32m     98 \u001b[0;31m            \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[0;32m---> 99 \u001b[0;31m        \u001b[0mdl_pack\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_dlpack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[0;32m    100 \u001b[0;31m        \u001b[0mtens\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfrom_dlpack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdl_pack\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[0;32m    101 \u001b[0;31m        \u001b[0;32mreturn\u001b[0m \u001b[0mtens\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "ipdb>  n\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MemoryError: std::bad_alloc: CUDA error at: ../include/rmm/mr/device/cuda_memory_resource.hpp:68: cudaErrorIllegalAddress an illegal memory access was encountered\n",
      "> \u001b[0;32m/jp_docs/nvtabular/nvtabular/loader/torch.py\u001b[0m(99)\u001b[0;36m_to_tensor\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32m     97 \u001b[0;31m        \u001b[0;32mif\u001b[0m \u001b[0mgdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mempty\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[0;32m     98 \u001b[0;31m            \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[0;32m---> 99 \u001b[0;31m        \u001b[0mdl_pack\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_dlpack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[0;32m    100 \u001b[0;31m        \u001b[0mtens\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfrom_dlpack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdl_pack\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[0;32m    101 \u001b[0;31m        \u001b[0;32mreturn\u001b[0m \u001b[0mtens\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "ipdb>  n\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--Return--\n",
      "None\n",
      "> \u001b[0;32m/jp_docs/nvtabular/nvtabular/loader/torch.py\u001b[0m(99)\u001b[0;36m_to_tensor\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32m     97 \u001b[0;31m        \u001b[0;32mif\u001b[0m \u001b[0mgdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mempty\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[0;32m     98 \u001b[0;31m            \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[0;32m---> 99 \u001b[0;31m        \u001b[0mdl_pack\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_dlpack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[0;32m    100 \u001b[0;31m        \u001b[0mtens\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfrom_dlpack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdl_pack\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[0;32m    101 \u001b[0;31m        \u001b[0;32mreturn\u001b[0m \u001b[0mtens\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "ipdb>  q\n"
     ]
    },
    {
     "ename": "BdbQuit",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mBdbQuit\u001b[0m                                   Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-12-3846398fef60>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      6\u001b[0m                                           \u001b[0moptimizer\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m                                           \u001b[0;31m#transform=batch_transform,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m                                           \u001b[0mamp\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      9\u001b[0m                                          )\n\u001b[1;32m     10\u001b[0m     \u001b[0mtrain_rmspe\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrmspe_func\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_pred\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/jp_docs/nvtabular/nvtabular/framework_utils/torch/utils.py\u001b[0m in \u001b[0;36mprocess_epoch\u001b[0;34m(dataloader, model, train, optimizer, loss_func, transform, amp, device)\u001b[0m\n\u001b[1;32m     48\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_grad_enabled\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     49\u001b[0m         \u001b[0my_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_pred_list\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 50\u001b[0;31m         \u001b[0;32mfor\u001b[0m \u001b[0midx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataloader\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     51\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mtransform\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     52\u001b[0m                 \u001b[0mx_cat\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx_cont\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtransform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/jp_docs/nvtabular/nvtabular/loader/backend.py\u001b[0m in \u001b[0;36m__next__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    263\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    264\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__next__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 265\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_next_batch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    266\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    267\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_fetch_chunk\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/jp_docs/nvtabular/nvtabular/loader/backend.py\u001b[0m in \u001b[0;36m_get_next_batch\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    290\u001b[0m         \u001b[0;31m# get the first chunks\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    291\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_batch_itr\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 292\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fetch_chunk\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    293\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    294\u001b[0m         \u001b[0;31m# try to iterate through existing batches\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/jp_docs/nvtabular/nvtabular/loader/backend.py\u001b[0m in \u001b[0;36m_fetch_chunk\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    269\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mchunks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    270\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 271\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mchunks\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    272\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_batch_itr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0miter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mchunks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    273\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/jp_docs/nvtabular/nvtabular/loader/backend.py\u001b[0m in \u001b[0;36mload_chunks\u001b[0;34m(self, dev, dataloader)\u001b[0m\n\u001b[1;32m    120\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    121\u001b[0m                         \u001b[0;31m# map from big chunk to fraemwork specific tensors\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 122\u001b[0;31m                         \u001b[0mchunks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdataloader\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_create_tensors\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mchunks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    123\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    124\u001b[0m                         \u001b[0;31m# split them into batches and map to\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/jp_docs/nvtabular/nvtabular/loader/torch.py\u001b[0m in \u001b[0;36m_create_tensors\u001b[0;34m(***failed resolving arguments***)\u001b[0m\n\u001b[1;32m    112\u001b[0m         \u001b[0mcats\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_to_tensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgdf_cats\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlong\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    113\u001b[0m         \u001b[0mconts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_to_tensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgdf_conts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloat32\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 114\u001b[0;31m         \u001b[0mlabel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_to_tensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgdf_label\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloat32\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    115\u001b[0m         \u001b[0;32mdel\u001b[0m \u001b[0mgdf_cats\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgdf_conts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgdf_label\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    116\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mcats\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabel\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/jp_docs/nvtabular/nvtabular/loader/torch.py\u001b[0m in \u001b[0;36m_to_tensor\u001b[0;34m(self, gdf, dtype)\u001b[0m\n\u001b[1;32m     97\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mgdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mempty\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     98\u001b[0m             \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 99\u001b[0;31m         \u001b[0mdl_pack\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_dlpack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    100\u001b[0m         \u001b[0mtens\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfrom_dlpack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdl_pack\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    101\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mtens\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.7/bdb.py\u001b[0m in \u001b[0;36mtrace_dispatch\u001b[0;34m(self, frame, event, arg)\u001b[0m\n\u001b[1;32m     90\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdispatch_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mframe\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0marg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     91\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mevent\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'return'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 92\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdispatch_return\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mframe\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0marg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     93\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mevent\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'exception'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     94\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdispatch_exception\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mframe\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0marg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.7/bdb.py\u001b[0m in \u001b[0;36mdispatch_return\u001b[0;34m(self, frame, arg)\u001b[0m\n\u001b[1;32m    152\u001b[0m             \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    153\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mframe_returning\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 154\u001b[0;31m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mquitting\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mBdbQuit\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    155\u001b[0m             \u001b[0;31m# The user issued a 'next' or 'until' command.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    156\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstopframe\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mframe\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstoplineno\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mBdbQuit\u001b[0m: "
     ]
    }
   ],
   "source": [
    "for epoch in range(EPOCHS):\n",
    "    start_train=time()\n",
    "    train_loss, y_pred, y = process_epoch(train_loader, \n",
    "                                          model, \n",
    "                                          train=True, \n",
    "                                          optimizer=optimizer,\n",
    "                                          #transform=batch_transform,\n",
    "                                          amp=False,\n",
    "                                         )\n",
    "    train_rmspe = rmspe_func(y_pred, y)\n",
    "    train_time=time() - start_train\n",
    "    y_pred = None\n",
    "    y = None\n",
    "    start_valid=time()\n",
    "    valid_loss, y_pred, y = process_epoch(valid_loader,\n",
    "                                          model, \n",
    "                                          train=False, \n",
    "                                          optimizer=optimizer,\n",
    "                                          #transform=batch_transform,\n",
    "                                          amp=False,\n",
    "                                         )\n",
    "    valid_rmspe = rmspe_func(y_pred, y)\n",
    "    valid_time = time() - start_valid\n",
    "    y_pred = None\n",
    "    y = None\n",
    "    print(f\"Train:{train_time} + Valid:{valid_time} = EpochTotal:{train_time + valid_time}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "file_extension": ".py",
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.8"
  },
  "mimetype": "text/x-python",
  "name": "python",
  "npconvert_exporter": "python",
  "pygments_lexer": "ipython3",
  "version": 3
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
